#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 27 14:13:13 2022

@author: cythnia
"""

#爬虫第6课：selenium工具包爬取异步加载网页（以贝壳网中杭州房价为例）
#selenium工具包:同时替代requests和Beautifulsoup的功能，模拟真实用户对浏览器进行操作
#——————————————————————————————————————————————————————————————————————————#
#导入工具包
import numpy as np
import pandas as pd
import time
from selenium import webdriver 
##设置driver为调用谷歌浏览器
driver=webdriver.Chrome()
#设置爬取的网址（此处为多页爬取，故观察到网页翻页规律后，直接设置列表）
url=['https://hz.fang.ke.com/loupan/pg{}/'.format(i) for i in range(1,11)] 
#生成空list存放数据
lis=[]
#for循环爬取数据
for urli in url:
    driver.get(urli) #获取网页信息
    driver.implicitly_wait(10) #获取到网页信息后，并不是马上进行爬取，而是等待十秒
    name=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-name > a') #爬取房子名称信息，element不加s为爬取单个数据，加s为爬取多个数据
    # for i in name:
    #     print(i.text)
    leixing=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-name > span:nth-child(3)')#爬取房子类型    
    # for i in leixing:
    #     print(i.text)
    zhuangtai=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-name > span.resblock-type')
    jiage=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li> div > div.resblock-price > div.main-price')#爬取价格信息
    zongjia=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-price > div.second')#爬取房子总价
    huxing=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > a.resblock-room')#爬取房子户型  
    guwen=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-agent')#爬取新房顾问
    guanjianci=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li> div > div.resblock-tag')#爬取关键词
    lianjie=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > div.resblock-name > a')#爬取链接
    dizhi=driver.find_elements_by_css_selector('body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li > div > a.resblock-location')#爬取地址
    #body > div.resblock-list-container.clearfix > ul.resblock-list-wrapper > li:nth-child(1) > div > a.resblock-location
    #汇总数据，for循环
    for names,leixings,zhuangtais,jiages,zongjias,huxings,guwens,guanjiancis,lianjies,dizhis in zip(name,leixing,zhuangtai,jiage,zongjia,huxing,guwen,guanjianci,lianjie,dizhi):
        namess=names.text
        leixingss=leixings.text
        zhuangtaiss=zhuangtais.text
        jiagess=jiages.text
        zongjiass=zongjias.text
        huxingss=huxings.text
        guwenss=guwens.text
        guanjianciss=guanjiancis.text
        lianjiess=lianjies.get_attribute('href')
        dizhiss=dizhis.text
        lis.append([namess,leixingss,zhuangtaiss,jiagess,zongjiass,huxingss,guwenss,guanjianciss,lianjiess,dizhiss])
    time.sleep(np.random.randint(5,10))
    print('完成爬取',urli)
result1=pd.DataFrame(lis) #生成二维数据
#修改列名，截取信息
result1.columns=['名称','类型','状态','价格/平方','总价','户型','顾问','卖点','链接','地址']
result1['总价']=result1['总价'].str.split('总价').str[1]
result1['建面']=result1['户型'].str.split('建面').str[1]
result1['户型']=result1['户型'].str.split('户型：').str[1]
result1['户型']=result1['户型'].str.split('建面').str[0]
result1['顾问']=result1['顾问'].str.split('新房顾问：').str[1]
result1
#将建面数据改位置
result_id = result1['建面']
result1 = result1.drop('建面',axis=1)
result1.insert(6,'建面',result_id)   
result1.to_excel('/Users/cythnia/Desktop/杭州楼盘房价.xlsx',index=False)
#————————————————————————————————————————————————————————————————————————————#
#dataFrame中insert的用法
#————————————————————————————————————————————————————————————————————————————#
# 用法:

# DataFrameName.insert(loc, column, value, allow_duplicates = False)

# 参数：

# loc:loc是一个整数，它是我们要插入新列的列的位置。这将使该位置上的现有列向右移动。

# column:column是一个字符串，它是要插入的列的名称。

# value:value只是要插入的值。它可以是int，string，float或任何东西，甚至可以是series /值列表。仅提供一个值将为所有行设置相同的值。

# allow_duplicates:allow_duplicates是一个布尔值，用于检查是否存在具有相同名称的列。

#————————————————————————————————————————————————————————————————————————————#
#爬取网页表格信息：pandas工具包
#————————————————————————————————————————————————————————————————————————————#
import pandas as pd
for table in pd.read_html(''):
    table.to_excel('')